In [1]:
import numpy as np
from sklearn.linear_model import LinearRegression
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
import seaborn as sns
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import cross_val_score
from sklearn.linear_model import LogisticRegression
from sklearn.decomposition import PCA
from sklearn import metrics
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
from scipy.stats import zscore
Load Table, Data pre-processing¶
In [2]:
cData = pd.read_csv("Vehicle.csv")
cData.shape
Out[2]:
In [3]:
cData.head()
Out[3]:
In [4]:
cData.describe().transpose()
Out[4]:
In [5]:
cData.dtypes
Out[5]:
In [6]:
cData.isnull().values.any()
Out[6]:
In [7]:
cData.isnull().sum()
Out[7]:
In [8]:
#Replace mising values
cData = cData.replace(' ', np.nan)
for i in cData.columns[:17]:
m = cData[i].median()
cData[i] = cData[i].fillna(m)
In [9]:
cData.isnull().values.any()
Out[9]:
Since variable 'Class' is category type:
In [10]:
cData['class']=cData['class'].astype('category')
cData.info()
Understanding Attributes¶
In [11]:
cData['class'].value_counts()
Out[11]:
In [12]:
sns.countplot(cData['class'])
Out[12]:
In [13]:
cData.hist(figsize=(18,18))
Out[13]:
In [14]:
#Outliers
g = cData.boxplot(figsize=(15,15))
g.set_xticklabels(plt.xticks()[1], rotation=60)
Out[14]:
In [15]:
for col_name in cData.columns[:-1]:
q1 = cData[col_name].quantile(0.25)
q3 = cData[col_name].quantile(0.75)
iqr = q3 - q1
low = q1-1.5*iqr
high = q3+1.5*iqr
cData.loc[(cData[col_name] < low) | (cData[col_name] > high), col_name] = cData[col_name].median()
g = cData.boxplot(figsize=(15,15))
g.set_xticklabels(plt.xticks()[1], rotation=60)
Out[15]:
In [16]:
labelencoder_X = LabelEncoder()
cData['class'] = labelencoder_X.fit_transform(cData['class'])
In [17]:
cData.corr()
Out[17]:
In [18]:
def plot_corr(df, size=14, title='Correlation'):
sns.set(font_scale=1.15)
fig,ax=plt.subplots(figsize=(size,size))
g = sns.heatmap(df.corr(),vmin=0.5, annot=True, linewidths=0.01,center=1,linecolor="white",cbar=False,square=True)
bottom, top = g.get_ylim()
g.set_ylim(bottom + 0.5, top - 0.5)
g.set_xticklabels(plt.xticks()[1], rotation=60)
plt.title(title,fontsize=12)
ax.tick_params(labelsize=14)
plot_corr(cData)
High Positive corelation exist between scatter_ratio, pr.axis_rectangularity, scaled_variance1, distance_circularity, scaled_variance, radius_ratio.
High negative corelation exist with elongatedness.
We can clrealy see that following features have very low corelation, as compared to others: pr.pr.axis_aspect_ratio, max.length_aspect_ratio, scaled_radius_of_gyration.1, skewness_about, skewness_about.1, skewness_about.2, hollows_ratio.
In [19]:
sns.pairplot(cData, hue='class')
Out[19]:
It clearly tell us: High Positive corelation exist between scatter_ratio, pr.axis_rectangularity, scaled_variance1, distance_circularity, scaled_variance, radius_ratio.
High negative corelation exist with elongatedness.
We can clrealy see that following features have very low corelation, as compared to others: pr.pr.axis_aspect_ratio, max.length_aspect_ratio, scaled_radius_of_gyration.1, skewness_about, skewness_about.1, skewness_about.2, hollows_ratio.
- 'compactness', 'distance_circularity', 'radius_ratio' and 'circularity' has negative linear relation with 'elongatedness'.
- 'compactness' has positive relation with 'circularity', 'distance_circularity', 'radius_ratio', 'scatter_ratio', 'pr.axis_rectangularity', 'max.axis_rectangularity', 'scaled_variance1', 'scaled_variance'.
- 'circularity' has positive relation with distance_circularity, scatter_ratio, pr.axis_rectangularity, max.axis_rectangularity, scaled_variance, scaled_variance1, scaled_radius_of_gyration.
- 'distance_circularity' has positive relation with 'radius_ratio', 'scatter_ratio, 'pr.axis_rectangularity', 'max.axis_rectangularity', 'scaled_variance, 'scaled_variance1, 'scaled_radius_of_gyration'.
- 'radius_ratio' has positive relation with 'pr.axis_aspect_ratio', 'scatter_ratio', 'scaled_variance', 'scaled_variance1', 'scaled_radius_of_gyration'.
In [ ]:
Split the data¶
In [20]:
x = cData.iloc[:,0:18]
y = cData.iloc[:,18]
# Split x and y into training and test set in 70:30 ratio
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.3, random_state = 10)
Logistic Regression Model¶
In [21]:
LogisticModel = LogisticRegression()
LogisticModel.fit(x_train, y_train)
LogisticModelPrediction = LogisticModel.predict(x_test)
model_score_Logistic = LogisticModel.score(x_test, y_test)
print(model_score_Logistic)
In [22]:
print('Accuracy on Training data: ',LogisticModel.score(x_train, y_train))
print('Accuracy on Testing data: ',LogisticModel.score(x_test , y_test))
print('Recall value: ',metrics.recall_score(y_test, LogisticModelPrediction, average='macro'))
print('Precision value: ',metrics.precision_score(y_test, LogisticModelPrediction, average='macro'))
print("Confusion Matrix:\n", metrics.confusion_matrix(y_test, LogisticModelPrediction))
print("classification Report:\n", metrics.classification_report(y_test, LogisticModelPrediction))
In [23]:
resultsDf1 = pd.DataFrame({'Model':['Logistic'],'Accuracy': LogisticModel.score(x_test , y_test)},index={'1'})
resultsDf1 = resultsDf1[['Model','Accuracy']]
resultsDf1
Out[23]:
GaussianNB Model¶
In [24]:
GaussianNBModel = GaussianNB()
GaussianNBModel.fit(x_train, y_train)
GaussianNBModelPrediction = GaussianNBModel.predict(x_test)
model_score_GaussianNB = GaussianNBModel.score(x_test, y_test)
print(model_score_GaussianNB)
In [25]:
print('Accuracy on Training data: ',GaussianNBModel.score(x_train, y_train))
print('Accuracy on Testing data: ', GaussianNBModel.score(x_test , y_test))
print('Recall value: ', metrics.recall_score(y_test, GaussianNBModelPrediction, average='macro'))
print('Precision value: ', metrics.precision_score(y_test, GaussianNBModelPrediction, average='macro'))
print("Confusion Matrix:\n", metrics.confusion_matrix(GaussianNBModelPrediction, y_test))
print("Classification Report:\n", metrics.classification_report(GaussianNBModelPrediction, y_test))
In [26]:
#Store the accuracy results for each kernel in a dataframe for final comparison
tempResultsDf = pd.DataFrame({'Model':['Naive Bayes'], 'Accuracy': GaussianNBModel.score(x_test, y_test)},index={'2'})
resultsDf1 = pd.concat([resultsDf1, tempResultsDf])
resultsDf1 = resultsDf1[['Model','Accuracy']]
resultsDf1
Out[26]:
Train a Support vector machine¶
In [27]:
svmClassifier = SVC()
svmClassifier = svmClassifier.fit(x_train, y_train)
svmClassifierPrediction = svmClassifier.predict(x_test)
In [28]:
print('Accuracy on Training data: ',svmClassifier.score(x_train, y_train))
print('Accuracy on Testing data: ', svmClassifier.score(x_test , y_test))
print('Recall value: ',metrics.recall_score(y_test, svmClassifierPrediction, average='macro'))
print('Precision value: ',metrics.precision_score(y_test, svmClassifierPrediction, average='macro'))
print("Confusion Matrix:\n",metrics.confusion_matrix(svmClassifierPrediction,y_test))
print("Classification Report:\n",metrics.classification_report(svmClassifierPrediction,y_test))
In [29]:
tempResultsDf = pd.DataFrame({'Model':['SVM'], 'Accuracy': svmClassifier.score(x_test, y_test)},index={'3'})
resultsDf1 = pd.concat([resultsDf1, tempResultsDf])
resultsDf1 = resultsDf1[['Model','Accuracy']]
resultsDf1
Out[29]:
Perform K-fold cross validation¶
In [30]:
#Use the Naive Bayes CLassifier with k fold cross validation
scores = cross_val_score(svmClassifier, cData, y, cv=18)
print(scores)
print('Average score: ', np.mean(scores))
In [31]:
#Store the accuracy results for each kernel in a dataframe for final comparison
tempResultsDf = pd.DataFrame({'Model':['SVM k fold'], 'Accuracy': np.mean(scores)}, index={'3'})
resultsDf1 = pd.concat([resultsDf1, tempResultsDf])
resultsDf1 = resultsDf1[['Model','Accuracy']]
resultsDf1
Out[31]:
In [ ]:
PCA¶
In [32]:
XScaled=x.apply(zscore)
XScaled.head()
Out[32]:
In [33]:
plt.rcParams['figure.figsize']=(10,6)
plt.plot(cData)
plt.show()
In [34]:
plt.rcParams['figure.figsize']=(10,6)
plt.plot(XScaled)
plt.show()
In [35]:
# Calculating the covariance between attributes after scaling
covMatrix = np.cov(XScaled.T, rowvar=True)
print('Covariance Matrix \n', covMatrix)
In [36]:
eigenvalues, eigenvectors = np.linalg.eig(covMatrix)
print('Eigen Vectors \n%s', eigenvectors)
print('\n Eigen Values \n%s', eigenvalues)
In [37]:
# Step 3 (continued): Sort eigenvalues in descending order
# Make a set of (eigenvalue, eigenvector) pairs
eig_pairs = [(eigenvalues[index], eigenvectors[:,index]) for index in range(len(eigenvalues))]
# Sort the (eigenvalue, eigenvector) pairs from highest to lowest with respect to eigenvalue
eig_pairs.sort()
eig_pairs.reverse()
print(eig_pairs)
# Extract the descending ordered eigenvalues and eigenvectors
eigvalues_sorted = [eig_pairs[index][0] for index in range(len(eigenvalues))]
eigvectors_sorted = [eig_pairs[index][1] for index in range(len(eigenvalues))]
# Let's confirm our sorting worked, print out eigenvalues
print('Eigenvalues in descending order: \n%s' %eigvalues_sorted)
In [38]:
tot = sum(eigenvalues)
var_explained = [( i /tot ) * 100 for i in sorted(eigenvalues, reverse=True)]
cum_var_exp = np.cumsum(var_explained)
print("Cumulative Variance Explained", cum_var_exp)
plt.plot(var_explained)
Out[38]:
In [39]:
# Ploting
plt.figure(figsize=(8 , 7))
plt.bar(range(1, eigenvalues.size + 1), var_explained, alpha = 0.5, align = 'center', label = 'Individual explained variance')
plt.step(range(1, eigenvalues.size + 1), cum_var_exp, where='mid', label = 'Cumulative explained variance')
plt.ylabel('Explained Variance Ratio')
plt.xlabel('Principal Components')
plt.legend(loc = 'best')
plt.tight_layout()
plt.show()
First 10 components catures arround 98.5% data, so we can eliminate last 7 components.¶
In [40]:
# Reducing from 17 to 10 dimension space
pca = PCA(n_components=10)
data_reduced = pca.fit_transform(XScaled)
data_reduced.transpose()
Out[40]:
In [41]:
pca.components_
Out[41]:
In [42]:
X_comp = pd.DataFrame(pca.components_,columns=list(XScaled))
X_comp.head()
Out[42]:
P_reduce represents reduced mathematical space. Reducing from 17 to 10 dimension space :¶
In [43]:
P_reduce = np.array(eigenvectors[0:10])
# projecting original data into principal component dimensions
X_std_10D = np.dot(XScaled,P_reduce.T)
# converting array to dataframe for pairplot
Proj_data_df = pd.DataFrame(X_std_10D)
sns.pairplot(Proj_data_df, diag_kind='kde')
Out[43]:
Maximum independet attributes have no/very low corelation. Some still have corelation and we have them here even after dimentionality reduction.¶
Build Models again¶
Splitting the data¶
In [44]:
# Split X and y into training and test set in 70:30 ratio
x_train1, x_test1, y_train1, y_test1 = train_test_split(Proj_data_df,y, test_size = 0.3, random_state = 10)
Logistic Regression Model¶
In [45]:
LogisticModel1 = LogisticRegression()
LogisticModel1.fit(x_train1, y_train1)
LogisticModelPrediction1 = LogisticModel1.predict(x_test1)
model_score_Logistic1 = LogisticModel1.score(x_test1, y_test1)
print(model_score_Logistic1)
In [46]:
print('Accuracy on Training data: ',LogisticModel1.score(x_train1, y_train1))
print('Accuracy on Testing data: ',LogisticModel1.score(x_test1 , y_test1))
print('Recall value: ',metrics.recall_score(y_test1, LogisticModelPrediction1, average='macro'))
print('Precision value: ',metrics.precision_score(y_test1, LogisticModelPrediction1, average='macro'))
print("Confusion Matrix:\n", metrics.confusion_matrix(y_test1, LogisticModelPrediction1))
print("classification Report:\n", metrics.classification_report(y_test1, LogisticModelPrediction1))
In [47]:
resultsDf2=pd.DataFrame({'Model':['Logistic'],'Accuracy': LogisticModel1.score(x_test1 , y_test1)},index={'1'})
resultsDf2=resultsDf2[['Model','Accuracy']]
resultsDf2
Out[47]:
GaussianNB Model¶
In [48]:
GaussianNBModel1 = GaussianNB()
GaussianNBModel1.fit(x_train1, y_train1)
GaussianNBModelPrediction1 = GaussianNBModel1.predict(x_test1)
model_score_GaussianNB1 = GaussianNBModel1.score(x_test1, y_test1)
print(model_score_GaussianNB1)
In [49]:
print('Accuracy on Training data: ',GaussianNBModel1.score(x_train1, y_train1))
print('Accuracy on Testing data: ', GaussianNBModel1.score(x_test1, y_test1))
print('Recall value: ', metrics.recall_score(y_test1, GaussianNBModelPrediction1, average='macro'))
print('Precision value: ', metrics.precision_score(y_test1, GaussianNBModelPrediction1, average='macro'))
print("Confusion Matrix:\n", metrics.confusion_matrix(GaussianNBModelPrediction1, y_test1))
print("Classification Report:\n", metrics.classification_report(GaussianNBModelPrediction1, y_test1))
In [50]:
#Store the accuracy results for each kernel in a dataframe for final comparison
tempResultsDf = pd.DataFrame({'Model':['Naive Bayes'], 'Accuracy': GaussianNBModel1.score(x_test1, y_test1)},index={'2'})
resultsDf2 = pd.concat([resultsDf2, tempResultsDf])
resultsDf2 = resultsDf2[['Model','Accuracy']]
resultsDf2
Out[50]:
Train a Support vector machine¶
In [51]:
svmClassifier1 = SVC()
svmClassifier1 = svmClassifier1.fit(x_train1, y_train1)
svmClassifierPrediction1 = svmClassifier1.predict(x_test1)
In [52]:
print('Accuracy on Training data: ',svmClassifier1.score(x_train1, y_train1))
print('Accuracy on Testing data: ', svmClassifier1.score(x_test1 , y_test1))
print('Recall value: ',metrics.recall_score(y_test1, svmClassifierPrediction1, average='macro'))
print('Precision value: ',metrics.precision_score(y_test1, svmClassifierPrediction1, average='macro'))
print("Confusion Matrix:\n",metrics.confusion_matrix(svmClassifierPrediction1,y_test1))
print("Classification Report:\n",metrics.classification_report(svmClassifierPrediction1,y_test1))
In [53]:
tempResultsDf = pd.DataFrame({'Model':['SVM'], 'Accuracy': svmClassifier1.score(x_test1, y_test1)},index={'3'})
resultsDf2 = pd.concat([resultsDf2, tempResultsDf])
resultsDf2 = resultsDf2[['Model','Accuracy']]
resultsDf2
Out[53]:
Perform K-fold cross validation¶
In [54]:
#Use the Naive Bayes CLassifier with k fold cross validation
scores = cross_val_score(svmClassifier1, Proj_data_df, y, cv=18)
print(scores)
print('Average score: ', np.mean(scores))
In [55]:
#Store the accuracy results for each kernel in a dataframe for final comparison
tempResultsDf = pd.DataFrame({'Model':['SVM k fold'], 'Accuracy': np.mean(scores)}, index={'3'})
resultsDf2 = pd.concat([resultsDf2, tempResultsDf])
resultsDf2 = resultsDf2[['Model','Accuracy']]
In [56]:
print("Before PCA:")
resultsDf1
Out[56]:
In [57]:
print("After PCA:")
resultsDf2
Out[57]:
We can conclude that:¶
- Before reducing attributes: Logistic Regression performed better
- After reducing attributes from 18 to 10
- SVM and SVM k fold performed bertter.
In [ ]:
In [ ]:
In [ ]: